package v2

import (
	"bufio"
	"fmt"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"
	"io/ioutil"
	"net/http"
	"time"
)

var rateLimiter = time.Tick(100 * time.Millisecond)

func Fetcher(url string) ([]byte,error) {
	<- rateLimiter
	req,_ := http.NewRequest(http.MethodGet,url,nil)
	var httpClient = http.Client{}
	rsp,err := httpClient.Do(req)
	if err != nil {
		return  nil,err
	}
	defer rsp.Body.Close()
	if rsp.StatusCode != 200 {
		return nil,fmt.Errorf("请求出错，返回码【%d】\n",rsp.StatusCode)
	}
	bufBody := bufio.NewReader(rsp.Body)
	utf8Reader := transform.NewReader(bufBody,determineEncoding(bufBody).NewDecoder())
	body,err := ioutil.ReadAll(utf8Reader)
	return body,err
}


func determineEncoding(r *bufio.Reader) encoding.Encoding {
	data,err := r.Peek(1024)
	if err != nil {
		return  unicode.UTF8
	}
	e,_,_ := charset.DetermineEncoding(data,"")
	return e
}